#!/usr/bin/perl

use strict;
use Data::Dumper;
use warnings;
use LWP::Simple;
use XML::Simple;
use URI::Escape;
use POSIX 'locale_h';
setlocale(LC_CTYPE, "no_NO") or warn "uh oh... $!";
use locale;
use Encode;

# Make these into options
my $repository = 'repository';
my $aliases = 'aliases';
my $kmltitle = 'db2kml.pl';
#my $kmlicon = 'http://optimus.nvg.org/webcam/a.png'; # asiaforum.no
my $kmlicon = 'http://www.asiaforum.no/favicon.ico'; # asiaforum.no
my $split = ":"; 

# read arguments
my $arg;
my $debug = 0;
my $test = 0;
my $populated = 0;
my $geon;
my $argnum;
my $dumpaliases = 0;
my $joomla = 0;

while(@ARGV) {
        $arg = shift(@ARGV);
	# print STDERR "ARG: $arg\n";
        if($arg eq '-repository') {
                $repository = shift;
        }
        if($arg eq '-aliasas') {
                $aliases = shift;
        }
        if($arg eq "-test") {
                $test = 1;
        }
        if($arg eq "-debug") {
                $debug = 1;
        }
        if($arg eq "-title") {
                $kmltitle = shift;
        }
        if($arg eq "-icon") {
                $kmlicon = shift;
        }
        if($arg eq "-split") {
                $split = shift;
        }
        if($arg eq "-populated") {
                $populated = 1;
        }
        if($arg eq "-geonames") {
                $geon = 1;
        }
        if($arg eq "-dumpaliases") {
                $dumpaliases = 1;
        }
	if($arg eq "-joomla") {
		$joomla = 1;
	}
}

# read repository into memory
my %newsitems = ();
print STDERR "Repository = $repository\n" if $debug;
print STDERR "Populated = $populated\n" if $debug;
open(REP, "<$repository") || die "Can not open $repository";
while(<REP>) {
        next if /^#/;
        chop;
        my($url,$title) = split(/-->>/);
        next if $url =~ /^\s*$/;
        next if $title =~ /^\s*$/;
        $newsitems{$url} = $title;
}
close(REP);

# read aliases into memory
my %aliases = ();
my %locations = ();
my %foundPositions = ();
open(ALIASES, "<$aliases") || die "Can not open $aliases";
while(<ALIASES>) {
        next if /^#/;
        chop;
        my($alias,$realname) = split(/:\s*/);
        next if $alias =~ /^\s*$/;
        next if $realname =~ /^\s*$/;
	if($realname =~ /^\s*(\d+\.\d+)\s+(\d+\.\d+)\s*$/) {
		$locations{lc($alias)} = $2 . "," . $1;
		$foundPositions{lc($alias)} = $2 . "," . $1;
	} elsif($realname =~ /^\s*(-*\d+\.\d+)\s*\,\s*(-*\d+\.\d+)\s*$/) {
		$locations{lc($alias)} = $1 . "," . $2;
		$foundPositions{lc($alias)} = $1 . "," . $2;
	} else {	
        	$aliases{$alias} = $realname;
	}
}
close(ALIASES);



# read locations into memory from STDIN
my %locType = ();
my %pc = ();

if(!$geon) { # no need to read in locations if we use geonames
	while(<>) {
		my @line = split(/\t/);
		my $lon = $line[4];
		my $lat = $line[3];
		my $fc =  $line[9]; # P = populated place
		my $pc = $line[11]; # population classification, 1 = most important, 5 = least important, NULL (no value) = not classified
		my $dim = $line[15]; # dimension
		my $fullname = $line[22];
		next if ($populated && $fc ne "P"); # skip the location if it is not a populated place and we only want those
		if(defined($locations{lc($fullname)})) {
			# pick the location with the highest PC
			print STDERR "Warning: duplicate name $fullname\n" if (defined($pc{lc($fullname)}) && $debug);
			next if !defined($pc{lc($fullname)}) || $pc =~ /^\s*$/; # it is empty, that is, not classified, so we keep the one we already have, first in wins :-)
			if ($pc{lc($fullname)} =~ /^\s*$/ || $pc < $pc{lc($fullname)}) {
				print STDERR "Changing $fullname from $pc{lc($fullname)} to $pc, dimension = $dim\n" if $debug; 
				$locations{lc($fullname)} = "$lon,$lat";
        	        	$locType{lc($fullname)} = $fc;
                		$pc{lc($fullname)} = $pc;
			}
		} else {
			$locations{lc($fullname)} = "$lon,$lat";
			$locType{lc($fullname)} = $fc;
			$pc{lc($fullname)} = $pc;
		}
	}
}

# Finne alle innlegg og behandle dem
my %foundLocations = ();
my %numberofarticles = ();
my $maxarticles = 0;
foreach my $item (keys %newsitems) {
	next unless defined($newsitems{$item});
    
	#find the preamble og the text and remove it
    	my($pre,@rest) = split($split, $newsitems{$item});
	my @words = ();
	if($#rest < 0) { # have no spilt so we keep all the text
		@words = split('\W', $newsitems{$item});	
	} else {
		$newsitems{$item} = join(' ', @rest);
		@words = split('\W', $newsitems{$item});
    		# put back the preamble
 		$newsitems{$item} = $pre .':' . $newsitems{$item};
    	}
	for(my $i=0;$i<=$#words;$i++) {
		for (my $t=$#words;$t>=$i;$t--) {
			my $word = join(" ", @words[$i .. $t]);
			# trim the words
			$word =~ s/^\s*(.+)\s*$/$1/;
			if(defined($aliases{lc($word)})) {
				$word = $aliases{lc($word)};
			}
			next if $word eq 'ignore';
			if($geon) {
				my($pos);
				if(!$foundPositions{lc($word)}) {
					$pos = lookup_geonames(lc($word));
				} else {
					$pos = $foundPositions{lc($word)};
				}
				if(defined($pos)) {
					$foundPositions{lc($word)} = $pos;	
					$foundLocations{lc($word)} .= addURL($newsitems{$item},$item);
					$numberofarticles{lc($word)}++;
					if($numberofarticles{lc($word)} > $maxarticles){
					    $maxarticles = $numberofarticles{lc($word)};
					}
					$i = $t;
				} else {
					$aliases{lc($word)} = 'ignore';
				}
			} elsif(defined($locations{lc($word)})) {
				$foundLocations{lc($word)} .= addURL($newsitems{$item},$item);
				$numberofarticles{lc($word)}++;
				if($numberofarticles{lc($word)} > $maxarticles){
				    $maxarticles = $numberofarticles{lc($word)};
				}
				$i = $t;
			}
		}
	}
}

if($dumpaliases) {
	open(ALIASES, ">$aliases") || die "Can not open $aliases";
	foreach my $alias (sort keys(%aliases)) {
		print ALIASES "$alias: $aliases{$alias}\n";
	}  
	foreach my $alias (sort keys(%foundPositions)) {
		print ALIASES "$alias: $foundPositions{$alias}\n";
	}  
	close(ALIASES);
}

if($joomla) {
	print "delete from jo_tj_google_maps where catid=15;\n";
	foreach my $place (sort keys(%foundLocations)) {
		$place =~ s/\b(\w)/uc($1)/eg;
		$foundLocations{lc($place)} =~ s/<a .+?>(.+?):.*?<\/a><br \/>/$1, /gis;
		$foundLocations{lc($place)} =~ s/\n//gis;
		$foundLocations{lc($place)} =~ s/\'/\\'/gis;
		chop($foundLocations{lc($place)});	
		chop($foundLocations{lc($place)});	
		print "insert into jo_tj_google_maps (name,misc,lng,lat,published,catid,user_id) values ('$place','" . $foundLocations{lc($place)} . 
"'," . $foundPositions{lc($place)} . ",1,15,64);\n";
	}
	exit(0);
}

kmlPre($kmltitle);

foreach my $place (sort keys(%foundLocations)) {
	print "<Placemark>\n";
	my $style = ($numberofarticles{lc($place)}/$maxarticles * 10);
	$style = int($style/2); 
	my $fc = $locType{$place};
	$place =~ s/\b(\w)/uc($1)/eg;
	my($name,$pos);
	print "<name>$place</name>\n";
	print "<styleUrl>#perlStyleID$style</styleUrl>\n";
	print "<description>\n<![CDATA[\n";
	print $foundLocations{lc($place)} . "\n";
	print "]]>\n</description>\n";
	if($geon) {
 		print "<Point><coordinates>$foundPositions{lc($place)},0</coordinates></Point>\n";
	} else {	
 		print "<Point><coordinates>$locations{lc($place)},0</coordinates></Point>\n";
	}
	print "</Placemark>\n";
}

kmlPost();

sub addURL {
	my($title,$url) = @_;
	$title =~ s/^.+\s::\s//;
	return "<a href=\"$url\">$title</a><br />\n";
}

sub kmlPre {
	my($title) = shift;
	my($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst)= localtime(time+86400);
	my($expire) = sprintf "%4d-%02d-%02dT%02d:%02d:%02dZ",$year+1900,$mon+1,$mday,$hour,$min,$sec;
	print <<KML;
<?xml version="1.0" encoding="UTF-8"?>
<kml xmlns="http://earth.google.com/kml/2.1">
<NetworkLinkControl>
<expires>$expire</expires>
</NetworkLinkControl>'
<Document>
  <Style id="perlStyleID0">
    <IconStyle>
      <scale>.75</scale>
        <Icon>
          <href>$kmlicon</href>
          <x>128</x>
          <y>64</y>
          <w>72</w>
          <h>73</h>
        </Icon>
    </IconStyle>
    <LabelStyle>
      <color>FFffaaff</color>
      <scale>.75</scale>
    </LabelStyle>
  </Style>
  <Style id="perlStyleID1">
    <IconStyle>
      <scale>.75</scale>
        <Icon>
          <href>$kmlicon</href>
          <x>128</x>
          <y>64</y>
          <w>72</w>
          <h>73</h>
        </Icon>
    </IconStyle>
    <LabelStyle>
      <color>FFffaaff</color>
      <scale>1</scale>
    </LabelStyle>
  </Style>
  <Style id="perlStyleID2">
    <IconStyle>
      <scale>.75</scale>
        <Icon>
          <href>$kmlicon</href>
          <x>128</x>
          <y>64</y>
          <w>72</w>
          <h>73</h>
        </Icon>
    </IconStyle>
    <LabelStyle>
      <color>FFffaaff</color>
      <scale>1.25</scale>
    </LabelStyle>
  </Style>
  <Style id="perlStyleID3">
    <IconStyle>
      <scale>.75</scale>
        <Icon>
          <href>$kmlicon</href>
          <x>128</x>
          <y>64</y>
          <w>72</w>
          <h>73</h>
        </Icon>
    </IconStyle>
    <LabelStyle>
      <color>FFffaaff</color>
      <scale>1.5</scale>
    </LabelStyle>
  </Style>
  <Style id="perlStyleID4">
    <IconStyle>
      <scale>.75</scale>
        <Icon>
          <href>$kmlicon</href>
          <x>128</x>
          <y>64</y>
          <w>72</w>
          <h>73</h>
        </Icon>
    </IconStyle>
    <LabelStyle>
      <color>FFffaaff</color>
      <scale>1.75</scale>
    </LabelStyle>
  </Style>
  <Style id="perlStyleID5">
    <IconStyle>
      <scale>.75</scale>
        <Icon>
          <href>$kmlicon</href>
          <x>128</x>
          <y>64</y>
          <w>72</w>
          <h>73</h>
        </Icon>
    </IconStyle>
    <LabelStyle>
      <color>FFffaaff</color>
      <scale>2</scale>
    </LabelStyle>
  </Style>

	<Folder>
		<name>$title</name>
	
KML
}

sub kmlPost {
	print <<KML;
	</Folder>
</Document>
</kml>
KML
}

sub lookup_geonames {
	my $location = shift;
	return undef if $location =~ /^\s*$/;
	return undef if $location =~ /[1234567890]/; 
	$location = encode("utf8", lc($location)); #make sure we use utf8 in the lookup 
	my $geonames = 'http://ws.geonames.org/search?maxRows=10&q=' . uri_escape($location);
	my $geonamesresponse = get($geonames);
	my $xmlsimple = XML::Simple->new();
	my $xml = $xmlsimple->XMLin($geonamesresponse, KeyAttr=>[], ForceArray => 1);
	print STDERR "navn: $location\nurl: $geonames\n" if $debug;	
	my($returned_items) = $xml->{totalResultsCount};
	print STDERR "items: $returned_items\n" if $debug;
	if(defined($returned_items) && $returned_items > 0) {
		print STDERR "Starter loop\n" if $debug;
		foreach my $geoname (@{$xml->{geoname}}) {
			print STDERR "$geoname\n" if $debug;
			my $name = encode("utf8", lc($geoname->{name}->[0]));
			my $lat = $geoname->{lat}->[0];
			my $lon = $geoname->{lng}->[0];
			print STDERR "$name - $lon,$lat\n" if $debug;
			if($name eq $location) {
				print STDERR "FOUND!\n" if $debug;
				return "$lon,$lat";
			}
		}
		print STDERR "Loop slutt\n" if $debug;
	} 
	return undef;
}
